use strict;
use utf8;

use MediaWiki::DumpFile::SQL;
use MediaWiki::DumpFile;
use Data::Dumper;
use Devel::Size qw(size total_size);

$Data::Dumper::Indent = 2;
$Data::Dumper::Useqq = 1;
{ no warnings 'redefine';
  sub Data::Dumper::qquote {
    my $s = shift;
    return "'$s'";
  }
}
 
my $category_fn = "zcat /home/lazyfrog-nb-ubuntu/data/ruwiki-20120213-category.sql.gz |";
my $categorylinks_fn = "< /home/lazyfrog-nb-ubuntu/data/ruwiki-20120213-categorylinks.sql";
#my $pages_fn = "bzcat /home/lazyfrog-nb-ubuntu/data/ruwiki-20120213-pages-articles.xml.bz2 |";
my $pages_fn = "< /home/lazyfrog-nb-ubuntu/data/ruwiki-20120213-pages-articles.xml";


binmode(STDOUT, ":encoding(utf-8)");
binmode(STDERR, ":encoding(utf-8)");

#if ($#ARGV >= 0) { 
#  my $dump_fn = $ARGV[0];
#  print "loading dump from $dump_fn ...\n";
#  open(F, "> $dump_fn");
#  binmode(F, ":encoding(utf-8)");
#  my $dump;
#  while (<F>) { $dump .= $_; }
#  close(F);
#  my $rr = eval $dump;
#  foreach my $a (keys %{$rr}) {
#    print $a . "\n";
#  }
#
#  process_cat_hier($rr);
#}
#die;

#open(C, "$category_fn") || die "can't open $category_fn";
#binmode(C, ":encoding(utf-8)");
#my $sql = MediaWiki::DumpFile::SQL->new(\*C);
#$sql->{type_map}->{varbinary} = "varchar";

#my %cat;
my %name2id;

#while (defined(my $row = $sql->next)) {
#  $cat{$row->{cat_id}} = $row;
  # Note: cat_id isn't a wiki page id !

#  $row->{cat_title} =~ s/\\\"/\"/g;
#  $name2id{$row->{cat_title}} = $row->{cat_id}->{title};
#  foreach my $k (keys %{$row}) { print STDERR "$row->{$k} "; } print STDERR "\n";
#}

#close(C);

#print "total_size(%cat) == " . total_size(\%cat) . "\n";
print "total_size(%name2id) == " . total_size(\%name2id) . "\n";

print "pagesarticles ...\n";

open(P, "$pages_fn") || die "can't open $pages_fn";
my $mw = MediaWiki::DumpFile->new;
my $pages = $mw->pages(\*P);
my $nc = 0;
my $max_id = 0;

while(defined(my $page = $pages->next)) {
  if ($page->id > $max_id) {
    $max_id = $page->id;
  }
last;

  if ($page->redirect) {
    print STDERR "PAGE " . $page->id . " " . $page->title . " is redirect\n";
    next;
  }

  print STDERR "PAGE " . $page->id . ' ' . $page->title . "\n";

#  if (exists $p{$page->id}) {
    #print STDERR $page->id . " " . $page->title . " exists in hash \"p\"\n";
#    if (exists $p{$page->id}->{title} && $p{$page->id}->{title} ne $page->title) {
#      print STDERR "ERROR: \"" . $p{$page->id}->{title} . "\" ne \"" . $page->title . "\"\n";
#    }

#    if ($page->title =~ /Категория:/) {
      my $title = $page->title;
      $title =~ s/\s/_/g;
      #$title =~ s/^Категория://;
      #p[$page->id]->[3] = $title;
      $name2id{$title} = $page->id;
#      if ($#{$p{$page->id}->{uplinks}} <= 0) {
#        print $page->id . " \"" . $title . "\" has " . $#{$p{$page->id}->{uplinks}} . " uplinks\n";
#        print "\t" . join("\n\t", @{$p{$page->id}->{uplinks}}) . "\n";
#      }
      #if ($nc > 100) { last; }
      #$nc += 1;
#    }
#  }
}

#print "total_size(%cat) == " . total_size(\%cat) . "\n";
print "total_size(%name2id) == " . total_size(\%name2id) . "\n";

print "categorylinks ...\n";

my @catlinks;
#my %p;
my @p;
$#p = 4000000;
# uplinks: $p{id}->[0]
# downlinks: $p{id}->[1]
# pages: $p{id}->[2]

open(F, "$categorylinks_fn") || die "can't open $categorylinks_fn";
binmode(F, ":encoding(utf-8)");
while (<F>) {
  if ($_ =~ /^INSERT INTO/) {
#    while ($_ =~ /\((\d+),\'([^\']+)\',\'([^\']*)\',\'([^\']*)\',\'([^\']*)\',\'([^\']*)\',\'([^\']*)\'\)/g) {
    while ($_ =~ /\((\d+),\'([^\']+)\',\'[^\']*\',\'[^\']*\',\'[^\']*\',\'[^\']*\',\'([^\']*)\'\)/g) {
      my ($from, $to, $type) = ($1, $2, $3);
#                  from  => $1,
#                  to => $2,
#                  sortkey => $3,
#                  cl_timestamp => $4,
#                  sortkey_prefix => $5,
#                  cl_collation => $6,
#                  type => $7 );
      $to =~ s/\\\"/\"/g;
      
      print STDERR join(" ", ($1, $2, $7)) . "\n";
#      if (! exists($name2id{$row{cl_to}})) {
#        print STDERR "\"$row{cl_to}\" not exist in category table\n";
#      }
      if ("subcat" eq $type) {
        $to = "Категория:" . $to;
      }

      my $to_id = undef;

      if (! exists($name2id{$to})) {
        print STDERR "Page \"$to\" is referenced from $from but doesn't exist. ";
        $max_id += 1;
        my $new_id = $max_id;
        my $target_id = $new_id;
        print STDERR "New id is \"$target_id\"\n";
        $name2id{$to} = $target_id;
        $to_id = $target_id;
      } else {
        $to_id = $name2id{$to};
      }

#      my $key = $row{from} . "-" . $row{to};
#      if (exists($catlinks{$key})) {
#        print STDERR "$key already exists in categorylinks table\n";
#      }
     
      if ("subcat" eq $type || "page" eq $type) { 
        my @a = ($from, $to_id, $type);
        push @catlinks, \@a;
        #$cat{$name2id{$row{cl_to}}}->{$row{cl_type}} = $key;
        ####push @{$p[$from]->[0]}, \@a;
        #push @{$p{$name2id{$row{cl_to}}}->{downlinks}}, $key;
      }
    }
  }
}
close(F);

#print "total_size(%cat) == " . total_size(\%cat) . "\n";
print "total_size(%name2id) == " . total_size(\%name2id) . "\n";
print "total_size(catlinks) == " . total_size(\@catlinks) . "\n";
print "total_size(p) == " . total_size(\@p) . "\n";

print "catlinks size is " . $#catlinks . "\n";
#print "p size is " . scalar(keys %p) . "\n";


foreach my $la (@catlinks) {
  my $l = (from => $la->[0], to => $la->[1], type => $la->[2]);
  my $target_id = $name2id{$l->{to}};

  if ("subcat" eq $l->{type}) {
    push @{$p[$target_id]->[1]}, $l->{from};
  } elsif ("page" eq $l->{type}) {
    push @{$p[$target_id]->[2]}, $l->{from};
  }
}

print "total_size(p) == " . total_size(\@p) . "\n";

my @page_array;
for (my $pid = 0; $pid < $#p; $pid++) {
  if (! defined $p[$pid]) { next; }
#foreach my $r (@p) {
  my @a = ($pid, $p[$pid]->[3], $#{$p[$pid]->[0]}, $#{$p[$pid]->[1]}, $#{$p[$pid]->[2]});
  #my @a = (, $r->[3], $r->[0], $r->[1], $r->[2]);
  push @page_array, \@a;
}

print "total_size(@page_array) == " . total_size(\@page_array) . "\n";

foreach my $r (sort {$b->[3] <=> $a->[3]} @page_array) {
  print "PA " . join(" ", @{$r}) . "\n";
}

#my %r;
#$r{pages} = \%p;
#$r{cat} = \%cat;
#$r{catlinks} = \%catlinks;

#open(PD, "> cattree.perl.dump");
#binmode(PD, ":encoding(utf-8)");
#print PD Dumper(\%r);
#close(PD);

sub process_cat_hier {
  my $r = shift;

  

}
